#!/usr/bin/perl

use strict;
use YAML::Syck;
use IO::File;
use Web::Scraper;
use URI;

my $log = '/etc/httpd/logs/access_log';
my $domain = 'http://example.com';

my $f = new IO::File();
$f->open($log);
my %url;
while(my $line = <$f>){
    if( $line =~ /[GET|POST]+\ (\S+)\ HTTP/ ){
        $url{$1} = $url{$1}?$url{$1}+1:1;
    }
}
$f->close;

my @rank;
my $count = 0;
for my $key ( reverse sort {$url{$a} <=> $url{$b}} keys %url){
    last if( $count >= 10 );
    if( $key !~ /\.(jpeg|jpg|gif|png|css|js|ico)$/ ){ 
        my $uri = $domain.$key;
        my $links = scraper {
           process 'title','title[]' => 'TEXT'; 
           process 'meta','description[]' =>
                 sub{
                     return $_->attr_get_i('content')
                       if($_->attr_get_i('name') =~ /description/i && $_->attr_get_i('content'));
                     return; 
                };      
           process 'meta','modified[]' =>
                 sub{
                     return $_->attr_get_i('content')
                       if($_->attr_get_i('name') =~ /modified/i && $_->attr_get_i('content'));
                     return; 
                };      
           result qw/title description modified/
        }->scrape(URI->new($uri));
        push @rank,{ 
                       link     => $uri,
                       title    => $links->{title}->[0],
                       body     => $links->{description}->[0],
                       date     => $links->{modified}->[0],
                   };
    }
}

my $output = {};
$output->{entry} = \@rank;

print YAML::Syck::Dump $output;

